/*

Created by : Clemence, August 2018

This Dofile : Insheets raw custom data to create different database:
			 
			- Flux 2-4 (exports) 1995-2010 : flux24_cm
			- Flux 1-3 1995- 2010: flux13_cm
			- Flux 1 and 2 with invoiced currency :2010- 2017 : x_cur
			- Flux 3 (EU impors) 2011 -2017 : m_deb
			- Flux 4 with under-threshold observation and buyers' ID 1995-2017 : x_all_new_ss

Outcome data are saved in $sourcepath 

*/



cd $sourcepath
clear 
g year=1995 

save $sourcepath\flux13_cm.dta, replace 
clear 
g year=1995 
save $sourcepath\flux24_cm.dta, replace

*1995 - 2010 flux 1,2,3,4
forvalue i=1995(1)2010{
insheet using "$rawpathcm\ClassiqueMensuel`i'.txt", delimiter(";") clear
preserve
keep if flux==1 | flux==3
append using $sourcepath\flux13_cm
save $sourcepath\flux13_cm, replace 
restore
preserve
keep if flux==2 | flux==4
append using $sourcepath\flux24_cm
save $sourcepath\flux24_cm, replace 
restore
}

use $sourcepath\flux24_cm, clear
drop year
rename an year
save $sourcepath\flux24_cm, replace
use $sourcepath\flux13_cm, clear
drop year
rename an year
save $sourcepath\flux13_cm, replace


clear 
g year=2011 
save $sourcepath\x_cur, replace 

*FLUX 1 = IMPORTS data, 2010-2017
forvalue i=2011(1)2017{
insheet using "$rawpathcur\FULL_FLUX1_V3_`i'.csv", delimiter(";") clear
append using $sourcepath\x_cur
save $sourcepath\x_cur, replace 
}

*FLUX 2 = EXPORTS data, 2010-2017
forvalue i=2011(1)2017{
insheet using "$rawpathcur\FULL_FLUX2_V2_`i'.csv", delimiter(";") clear
append using $sourcepath\x_cur
save $sourcepath\x_cur, replace 
} 
rename valstat euros
drop year
rename an year
save $sourcepath\x_cur, replace

*FLUX 3 : IMPORT DEB, 2010-2017
cd $datapath 
clear 
g year=2011 
save $sourcepath\m_deb, replace 
forvalue i=2011(1)2017{
insheet using "$rawpathdeb\FULL_FLUX3_V1_`i'.csv", delimiter(";") clear
append using $sourcepath\m_deb
save $sourcepath\m_deb, replace 
}
rename valstat euros
drop year
rename an year
save $sourcepath\m_deb, replace 


*Flux 4 : Export DEB, with buyers ans under-threshold observations

clear 
g year= 1993

save $sourcepath\x_all_new_ss, replace 
forvalue i=1993(1)2017{
insheet using "$rawpathbuy\anonymV2Full_mois_`i'.csv", delimiter(";") clear
append using $sourcepath\x_all_new_ss
save $sourcepath\x_all_new_ss, replace 
}
gen ss=0
save $sourcepath\x_all_new_ss, replace 
clear
forvalue i=1993(1)2017{
import delimited "$rawpathbuy\anonymSousSeuils`i'.csv", delimiter(";") clear
rename payspart pyod
gen ss=1
append using $sourcepath\x_all_new_ss
save $sourcepath\x_all_new_ss, replace 
} 
use $sourcepath\x_all_new_ss, clear
gen flux2=4 if flux=="E"
drop flux
rename flux2 flux
drop year
rename an year
save $sourcepath\x_all_new_ss, replace


